import re
import requests
import pymongo
from fake_useragent import UserAgent
import xlwt


ua = UserAgent(use_cache_server=True)


class QS(object):
    headers = {'User-Agent': '{}'.format(ua.random)}
    client = pymongo.MongoClient("localhost")
    db = client['qishu']

    def __init__(self):
        self.i = 1
        self.workbook = xlwt.Workbook(encoding='utf-8')
        self.sheet1 = self.workbook.add_sheet("小说", cell_overwrite_ok=True)
        self.row0 = ["title", "click_number", "size", "download_type", "time", "status", "author_name"]
        for h in range(0, 7):
            self.sheet1.write(0, h, self.row0[h])

    def get_main_html(self):
        response = requests.get('https://www.qisuu.la/soft/sort01/', headers=self.headers).text
        pattern = re.findall(re.compile(r'<div class="s">(.*?)<br.*?</em><br>(.*?)</div>.*?<a href="(.*?)">', re.S), response)
        for detail_url in pattern:
            self.get_detail_url(detail_url[2])

    def get_detail_url(self, detail_url):
        new_url = "https://www.qisuu.la"+detail_url
        response = requests.get(new_url, headers=self.headers)
        response.encoding = "utf-8"
        pattern = re.findall(re.compile(r'<div class="detail_right">.*?<h1>(.*?)</h1>.*?<ul>.*?<li class="small">点击次数：(.*?)</li>.*?<li class="small">文件大小：(.*?)</li>.*?<li class="small">书籍类型：(.*?)</li>.*?<li class="small">更新日期：(.*?)</li>.*?<li class="small">连载状态：(.*?)</li>.*?<li class="small">书籍作者：(.*?)</li>', re.S), response.text)
        for content in pattern:
            print(content)
            print(pattern)
            self.save_to_excel(content)
            # self.create_dict(content)

    def create_dict(self, content):
        data_dict = {}
        data_dict['tile'] = content[0]
        data_dict['click_number'] = content[1]
        data_dict['size'] = content[1]
        data_dict['download_type'] = content[3]
        data_dict['time'] = content[4]
        data_dict['status'] = content[5]
        data_dict['author_name'] = content[6]
        self.db['qishu'].insert_one(data_dict)

    def save_to_excel(self, content):
        try:
            # colum0 = ["content[0]", "content[1]", "content[2]", "content[3]", "content[4]", "content[5]", "content[6]", "content[0]"]
            self.sheet1.write(self.i, 0, content[0])
            self.sheet1.write(self.i, 1, content[1])
            self.sheet1.write(self.i, 2, content[2])
            self.sheet1.write(self.i, 3, content[3])
            self.sheet1.write(self.i, 4, content[4])
            self.sheet1.write(self.i, 5, content[5])
            self.sheet1.write(self.i, 6, content[6])
            self.i += 1
            self.workbook.save("E:/text.xls")
            print("写入excel成功")
        except Exception as e:
            print("写入excel失败，error{}".format(e))


if __name__ == '__main__':
    qs = QS()
    qs.get_main_html()



